from twisted.internet import reactor, defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from week1.spiders import day01
from week1.spiders import day02
from week1.spiders import day03
from scrapy.utils.project import get_project_settings



# 打印HOST
from week1.settings import HOST


print(HOST)
# 打印所有settings配置
settings = get_project_settings()
print(dict(settings))


# 按顺序启动多个爬虫
configure_logging()
runner = CrawlerRunner(get_project_settings())
@defer.inlineCallbacks
def crawl():
    yield runner.crawl(day01.Day01Spider)
    # yield runner.crawl(day02.Day02Spider)
    # yield runner.crawl(day03.Day03Spider)
    reactor.stop()


crawl()
reactor.run()  # the script will block here until the last crawl call is finished
